package com.uusee.crawler.pageprocessor.video.tudou;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.HttpVersion;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.util.DomainUtils;
import com.uusee.crawler.util.PageProcessorUtils;
import com.uusee.crawler.util.PathUtils;
import com.uusee.crawler.xmlwriter.UgcXmlWriter;
import com.uusee.shipshape.sp.model.Ugc;
import com.uusee.util.StringUtils;

public class HdTudouComProgramPlayPageProcessor {
	private static final String A_REG="<a.*?>[\\s]*?([\\S].*?)</a>";
	private static Log log = LogFactory.getLog("HdTudouComProgramPlayPageProcessor");
	private Properties regex;
	Map<String,String> channelMap = new HashMap<String,String>();
	Map<String,String> categoryMap = new HashMap<String,String>();
	private static ThreadPoolExecutor producerPool = new ThreadPoolExecutor(5, 20, 0, TimeUnit.SECONDS,
			new ArrayBlockingQueue(10000), new ThreadPoolExecutor.CallerRunsPolicy());
	
	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("applicationContext.xml");
			Properties processRegex = (Properties)acx.getBean("tudouHDPlayPageProcessRegex");			
			Map<String,String> channelMap = (Map)acx.getBean("tudouHDChannelMap");
			Map<String,String> categoryMap = (Map)acx.getBean("tudouHDCategoryMap");
			
			HdTudouComProgramPlayPageProcessor processor = new HdTudouComProgramPlayPageProcessor();
			processor.regex = processRegex; 
			processor.categoryMap=categoryMap;
			processor.channelMap=channelMap;
			
			HttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
			HttpConnectionManagerParams hcmp = cm.getParams();
			hcmp.setDefaultMaxConnectionsPerHost(7);
			hcmp.setMaxTotalConnections(10);
			hcmp.setStaleCheckingEnabled(true);

			HttpClientParams hcp = new HttpClientParams();
			hcp.setVersion(HttpVersion.HTTP_1_0);

			HttpClient httpClient = new HttpClient(hcp, cm);
			StopWatch stopWatch = new StopWatch();
			stopWatch.start();
			
			for (int i=535;i<30000;i++) {
				try {
					int actionCount = producerPool.getActiveCount();
					while (actionCount > 4) {
						Thread.sleep(2000);
						actionCount = producerPool.getActiveCount();
					}
					String playUrl = "http://hd.tudou.com/program/"+i+"/";
					Ugc ugc = new Ugc();
					ugc.setPlaylistVid(i+"");
					ugc.setVideoSite("hd.tudou.com");
					ugc.setPlayUrl(playUrl);
					
					String filename = i+".xml";
					GetMethod get = new GetMethod(playUrl);
					get.setFollowRedirects(false);
					get.setRequestHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0;)");
					get.setRequestHeader("From", "abc@gmail.com");
					GetThread mpct = new GetThread(httpClient, get, ugc,filename,processor);
					producerPool.execute(mpct);
				} catch (Exception e) {
					e.printStackTrace();
					log.error(e);
				}

			}
			log.info("任务分配完成，等待结束。");
			while(true) {
				log.info("完成任务数："+producerPool.getCompletedTaskCount());
				try {
					Thread.sleep(20000);
				} catch (InterruptedException e) {
				}
				if (producerPool.getActiveCount()==0) {
					stopWatch.stop();
					System.out.println("抓取结束,用时："+stopWatch.getTime()/1000 + "秒。");
					System.exit(-1);
				}
			}
		}catch(Exception e){
			
		}
	}
	
	public boolean innerProcess(Ugc ugc,String pageHtml) {
		if (log.isDebugEnabled()) {
			log.debug("进入 innerProcess(Ugc,String) 方法。");
		}

		Date crawlDate = new Date();
		String playUrl = ugc.getPlayUrl();
		if (StringUtils.isEmpty(pageHtml)) {
			return false;
		}
		try {
			String videoText = pageHtml;
			String domain = DomainUtils.getDomain(playUrl);
			String sourceSite = PageProcessorUtils.getValue("http://([^/]*?)/", playUrl);
			String title = PageProcessorUtils.getValue(regex.getProperty("title"), videoText);
			if (StringUtils.isEmpty(title)) {
				return false;
			}
			
			title = StringUtils.replace(title, "<[^<].*?>", "");
			String logo = PageProcessorUtils.getValue(regex.getProperty("logo"), videoText);
			String slength = PageProcessorUtils.getValue(regex.getProperty("length"), videoText);
			int length = -1;
			try {
				length = Integer.valueOf(slength)*60;
			}catch(Exception e){
			}
			String channelId=PageProcessorUtils.getValue(regex.getProperty("channelId"), videoText);
			String channel="";
			if (channelMap.containsKey(channelId)) {
				channel = channelMap.get(channelId);
			}
			String categories = PageProcessorUtils.getValue(regex.getProperty("categoryIds"), videoText);
			String[] categoryIdArray = categories.split("\\|");
			StringBuffer categorysSb = new StringBuffer();
			String categorys = "";
			for (String categoryId:categoryIdArray) {
				if (categoryMap.containsKey(categoryId)) {
					String category = categoryMap.get(categoryId);
					categorysSb.append(category);
					categorysSb.append(",");
				}
			}
			if (categorysSb.length()>0) {
				categorys = categorysSb.substring(0, categorysSb.length()-1);
			}
			String tags = PageProcessorUtils.getValue(regex.getProperty("tags"), videoText);
			tags=PageProcessorUtils.getValues(A_REG, tags);
			String summary = PageProcessorUtils.getValue(regex.getProperty("summary"), videoText);
			String vid = PageProcessorUtils.getValue(regex.getProperty("vid"), videoText);
			String evid = PageProcessorUtils.getValue(regex.getProperty("evid"), videoText);
			String shareUrl = "http://www.tudou.com/v/"+evid;
			String sTotalPageView=PageProcessorUtils.getValue(regex.getProperty("totalPageView"), videoText);
			int totalPageView = 0;
			try {
				totalPageView = Integer.valueOf(sTotalPageView);
			}catch(Exception e){
			}
			
			String playlistVid=PageProcessorUtils.getValue(regex.getProperty("playlistVid"), videoText);
			String playlistId=PageProcessorUtils.getValue(regex.getProperty("playlistId"), videoText);
				
			ugc.setVideoSite("hd.tudou.com");
			ugc.setDomain(domain);
			ugc.setSourceSite(sourceSite);
			ugc.setPlayUrl(playUrl);
			ugc.setTitle(title);
			ugc.setLogo(logo);
			ugc.setLength(length);
			ugc.setChannel(channel);
			ugc.setCategory(categorys);
			ugc.setTags(tags);
			ugc.setSummary(summary);
			ugc.setVid(vid);
			ugc.setEvid(evid);
			ugc.setOriLogo(logo);
			ugc.setShareUrl(shareUrl);
			ugc.setTotalPageView(totalPageView);
			//ugc.setTotalComment(0);
			//ugc.setTotalFav(0);
			//ugc.setFileExt(fileExt);
			//ugc.setFileSize(fileSize);
			//ugc.setUploadUserid(uploadUserid);
			//ugc.setUploadUsername(uploadUsername);	
			//ugc.setUploadUserblog(uploadUserblog);
			//ugc.setUploadDate();
			ugc.setPlaylistVid(playlistVid);
			ugc.setPlaylistId(playlistId);
			ugc.setLogoStatus("0");
			ugc.setStatus("1");
			ugc.setCrawlUser("uusee_crawler");
			ugc.setCrawlDate(new Date());
			ugc.setUpdateUser("uusee_crawler");
			ugc.setUpdateDate(new Date());
			

			return true;
		} catch (Exception e) {
			e.printStackTrace();
			log.error("Analyze "+playUrl+" ERROR."+e);
			return false;
		}
	}
	
	static class GetThread extends Thread {
		private Lock lock = new ReentrantLock();
		private HdTudouComProgramPlayPageProcessor processor;
		private UgcXmlWriter writer = new UgcXmlWriter();
		private HttpClient httpClient;
		private GetMethod method;
		private Ugc ugc;
		private String filename;
		private String id;

		public GetThread(HttpClient httpClient, GetMethod method, Ugc ugc,String filename,HdTudouComProgramPlayPageProcessor processor) {
			this.httpClient = httpClient;
			this.method = method;
			this.ugc = ugc;
			this.filename = filename;
			this.id = ugc.getPlaylistVid();
			this.processor = processor;
		}

		/**
		 * Executes the GetMethod and prints some satus information.
		 */
		public void run() {
			BufferedOutputStream output = null;
			try {
				if (isExistData(ugc)) {
					log.info(id + " - 数据已经存在。");
					//return;
				}
				log.debug(id + " - 准备下载,数据地址:" + method.getURI());
				// execute the method
				int statusCode = httpClient.executeMethod(method);
				log.debug(id + " - 建立连接,状态码：" + statusCode);
				if (statusCode == HttpStatus.SC_OK) {
					StringBuilder sb = new StringBuilder();
					InputStream in = method.getResponseBodyAsStream();
					BufferedReader fIn = new BufferedReader(new InputStreamReader(in, "utf-8"));
					String l = fIn.readLine();
					while (l != null) {
						sb.append(l);
						l = fIn.readLine();
					}
					String htmlText = sb.toString();
					log.debug(id + " - 下载成功.");
					boolean flag = processor.innerProcess(ugc,htmlText);
					if (flag) {
						getDataFile(ugc);
						//writer.innerProcess(ugc);
						log.info(id + " - 生成XML。");
					}else{
						log.info(id+"-数据不可用。");
						return;
					}
				} else {
					log.info(id + " - 数据不存在");
				}
			} catch (Exception e) {
				log.error(id + " - 下载失败。error: " + e);
			} finally {
				if (output != null) {
					try {
						output.close();
					} catch (Exception e) {
					}
				}
				// always release the connection after we're done
				method.releaseConnection();
				log.debug(id + " - 释放链接。");
			}
		}

		public boolean isExistData(Ugc ugc) {
			String sourceSite = ugc.getVideoSite();
			String vid = ugc.getPlaylistVid();
			StringBuffer imagePathSb = new StringBuffer();
			imagePathSb.append("/data0/output/"+sourceSite+"/video/");
			imagePathSb.append(PathUtils.getContainIdSubPathById(vid));
			File imageFile = new File(imagePathSb.toString() + filename);
			if (imageFile.exists()) {
				return true;
			} else {
				return false;
			}
		}

		public File getDataFile(Ugc ugc) {
			String sourceSite = ugc.getVideoSite();
			String oriId = ugc.getPlaylistVid();
			StringBuffer imagePathSb = new StringBuffer();
			imagePathSb.append("/data0/output/"+sourceSite+"/video/");
			imagePathSb.append(PathUtils.getContainIdSubPathById(oriId));
			File imagePath = new File(imagePathSb.toString());
			try {
				lock.lock();
				if (!imagePath.exists()) {
					imagePath.mkdirs();
				}
				lock.unlock();
			} catch (Exception e) {
				lock.unlock();
			}
			File imageFile = new File(imagePath, filename);
			return imageFile;
		}
	}

}
